{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f299b647",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import json\n",
    "import time\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import os\n",
    "from torchvision import transforms, models, datasets\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import ssl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf19c2dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 是否继续训练模型的参数\n",
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b0134574",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_device() -> torch.device:\n",
    "    # 是否用GPU训练\n",
    "    train_on_gpu = torch.cuda.is_available()\n",
    "    \n",
    "    if not train_on_gpu:\n",
    "        print('CUDA is not available.  Training on CPU ...')\n",
    "    else:\n",
    "        print('CUDA is available!  Training on GPU ...')\n",
    "\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    return device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f9d56ab0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选择迁移哪个模型\n",
    "def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):\n",
    "    # 选择合适的模型，不同模型的初始化方法稍微有点区别\n",
    "    model_ft = None\n",
    "    input_size = 0\n",
    "\n",
    "    if model_name == \"resnet\":\n",
    "        \"\"\" \n",
    "        Resnet152\n",
    "        \"\"\"\n",
    "#         # pretrained 表示是否需要下载\n",
    "#         model_ft = models.resnet50(pretrained=False)  #   50\n",
    "        \n",
    "        model_ft = models.resnet50(pretrained = False)\n",
    "\n",
    "        pre=torch.load(r'./checkpoints50/resnet50-0676ba61.pth')  # 因不能下载\n",
    "        model_ft.load_state_dict(pre)\n",
    "        \n",
    "        \n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.fc.in_features\n",
    "        model_ft.fc = nn.Sequential(nn.Linear(num_ftrs, num_classes),nn.LogSoftmax(dim=1))\n",
    "        input_size = 224\n",
    "    else:\n",
    "        print(\"Invalid model name, exiting...\")\n",
    "        exit()\n",
    "\n",
    "    return model_ft, input_size\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b7501f66",
   "metadata": {},
   "outputs": [],
   "source": [
    "def lab_start():\n",
    "    # 种类数\n",
    "    num_classes = 24\n",
    "    model_ft, input_size = initialize_model(\"resnet\", num_classes, feature_extract=True, use_pretrained=True)\n",
    "    device = get_device()\n",
    "    model_ft = model_ft.to(device)\n",
    "    # 带name的params\n",
    "    params = model_ft.named_parameters()\n",
    "    print(\"Params need to learn:\")\n",
    "    params_need_update = []\n",
    "    for param_name, param in params:\n",
    "        if param.requires_grad:\n",
    "            params_need_update.append(param)\n",
    "            print(param_name)\n",
    "    #\n",
    "    #####################   文件夹          ##############################################\n",
    "    #\n",
    "    data_dir = './pics5.0/'\n",
    "    train_dir = data_dir + '/train'\n",
    "    valid_dir = data_dir + '/valid'\n",
    "\n",
    "    # 数据增强\n",
    "    data_transforms = {\n",
    "        'train': transforms.Compose([transforms.RandomRotation(25),  # 随机旋转，\n",
    "#                                      transforms.Resize(256), # 重新固定大小 #########\n",
    "#                                      transforms.CenterCrop(224),  # 从中心开始裁剪，只得到一张图片\n",
    "                                     transforms.RandomHorizontalFlip(p=0.5),  # 随机水平翻转 概率为0.5\n",
    "                                     transforms.RandomVerticalFlip(p=0.5),  # 随机垂直翻转\n",
    "                                     transforms.ColorJitter(brightness=0.2, contrast=0.1, saturation=0.1, hue=0.1),\n",
    "                                     # 参数1为亮度，参数2为对比度，参数3为饱和度，参数4为色相\n",
    "                                     transforms.RandomGrayscale(p=0.025),  # 概率转换成灰度率，3通道就是R=G=B\n",
    "                                     transforms.ToTensor(),\n",
    "                                     # 迁移学习，用别人的均值和标准差\n",
    "                                     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])  # 均值，标准差\n",
    "                                     ]),\n",
    "        'valid': transforms.Compose([\n",
    "                \n",
    "#                                      transforms.CenterCrop(224),\n",
    "                                     transforms.ToTensor(),\n",
    "                                     # 预处理必须和训练集一致\n",
    "                                     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "                                     ]),\n",
    "    }\n",
    "    ############################################################################\n",
    "    batch_size = 16\n",
    "\n",
    "    # train和valid的图片，做transform之后用字典保存\n",
    "    image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in\n",
    "                      ['train', 'valid']}\n",
    "\n",
    "    print(datasets.ImageFolder(os.path.join(data_dir, 'train'), data_transforms['train']).classes)\n",
    "    # 批量处理，这里都是tensor格式（上面compose）\n",
    "    dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True) for x in\n",
    "                   ['train', 'valid']}\n",
    "    # fuc = datasets.ImageFolder(os.path.join(data_dir, 'train'),data_transforms['train']).class_to_idx\n",
    "    # with open('fuc_index.txt', 'w') as file:\n",
    "    #     file.write(str(fuc))\n",
    "    # print(fuc)\n",
    "\n",
    "\n",
    "    dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'valid']}\n",
    "    print(\"dataset_sizes below:\")\n",
    "    print( dataset_sizes)\n",
    "    # 样本数据的标签\n",
    "    class_names = image_datasets['train'].classes\n",
    "    print(class_names)\n",
    "    ##############################################################################################\n",
    "    # 优化器设置\n",
    "    optimizer_ft = optim.Adam(params_need_update, lr=0.02)\n",
    "    # 学习率衰减\n",
    "    scheduler = optim.lr_scheduler.StepLR(optimizer_ft, step_size=4, gamma=0.8)  # 学习率每7个epoch衰减成原来的1/10\n",
    "    # 最后一层已经LogSoftmax()了，所以不能nn.CrossEntropyLoss()来计算了，nn.CrossEntropyLoss()相当于logSoftmax()和nn.NLLLoss()整合\n",
    "    criterion = nn.NLLLoss()  # 可以尝试改成交叉熵损失函数0702\n",
    "    ################################################################################\n",
    "    filename = \"./checkpoints50/checkpoints_resnet50_0812_5.0_01.pth\"\n",
    "\n",
    "    model_ft, val_acc_history, train_acc_history, valid_losses, train_losses, LRs = wz_model_train(model_ft,\n",
    "                                                                                                   dataloaders,\n",
    "                                                                                                   criterion,\n",
    "                                                                                                   optimizer_ft,\n",
    "                                                                                                   scheduler,\n",
    "                                                                                                   filename,\n",
    "                                                                                                   device)\n",
    "    for param in model_ft.parameters():\n",
    "        param.requires_grad = True\n",
    "\n",
    "    # 再继续训练所有的参数，学习率调小一点###########################################################################\n",
    "    optimizer = optim.Adam(params_need_update, lr=1e-3)\n",
    "    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)\n",
    "\n",
    "    # 损失函数\n",
    "    criterion = nn.NLLLoss()\n",
    "    checkpoint = torch.load(filename)\n",
    "    best_acc = checkpoint['best_acc']\n",
    "    model_ft.load_state_dict(checkpoint['state_dict'])\n",
    "    optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "\n",
    "    model_ft, val_acc_history, train_acc_history, valid_losses, train_losses, LRs = wz_model_train(model_ft,\n",
    "                                                                                                   dataloaders,\n",
    "                                                                                                   criterion,\n",
    "                                                                                                   optimizer,\n",
    "                                                                                                   scheduler, filename,\n",
    "                                                                                                   device,\n",
    "                                                                                                   num_epochs=25)  # 再训练5个纪元\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "05a0dd3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#############################################################################################################\n",
    "def wz_model_train(model, dataloaders, criterion, optimizer, scheduler, filename: str, device: torch.device,\n",
    "                   num_epochs=121, is_inception=False):  # 原来为 10 ，现在为25 ,num_epochs 为 训练的大阶段，共几个阶段\n",
    "    start_time = time.time()\n",
    "    best_acc = 0\n",
    "    best_model_weights = copy.deepcopy(model.state_dict())\n",
    "    model.to(device)\n",
    "    # 保存损失和准确率数据\n",
    "    val_acc_history = []\n",
    "    train_acc_history = []\n",
    "    train_losses = []\n",
    "    valid_losses = []\n",
    "    # 记录每个epoch的learningRate\n",
    "    LRs = [optimizer.param_groups[0]['lr']]\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # 训练和验证\n",
    "        for phase in ['train', 'valid']:\n",
    "            if phase == 'train':\n",
    "                model.train()  # 训练\n",
    "            else:\n",
    "                model.eval()  # 验证\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 清零\n",
    "                optimizer.zero_grad()\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # inception会有辅助输出，损失函数为一个线性模型\n",
    "                    if is_inception and phase == 'train':\n",
    "                        outputs, aux_outputs = model(inputs)\n",
    "                        loss1 = criterion(outputs, labels)\n",
    "                        loss2 = criterion(aux_outputs, labels)\n",
    "                        loss = loss1 + 0.4 * loss2\n",
    "                    else:  # resnet执行的是这里\n",
    "                        outputs = model(inputs)\n",
    "                        loss = criterion(outputs, labels)\n",
    "\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    # 训练阶段更新权重\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                # 计算损失\n",
    "                # loss计算默认都是取mean，计算批量的loss时，要乘以loss的数量\n",
    "                # 所以这里计算的是一个epoch里所有样本的loss和正确数量\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "            # 完整一次的loss均值和准确率\n",
    "            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n",
    "            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "            # 一个epoch里train和valid分别花的时间和loss和准确度\n",
    "            time_elapsed = time.time() - start_time\n",
    "            print('Time elapsed {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "            # 得到最好那次的模型\n",
    "            if phase == 'valid' and epoch_acc > best_acc:  # 最大如何比较 0624:就是单纯数值比较0629\n",
    "                best_acc = epoch_acc\n",
    "                best_model_weights = copy.deepcopy(model.state_dict())\n",
    "                state = {\n",
    "                    'state_dict': model.state_dict(),\n",
    "                    'best_acc': best_acc,\n",
    "                    'optimizer': optimizer.state_dict(),\n",
    "                }\n",
    "                torch.save(state, filename)\n",
    "            if phase == 'valid':\n",
    "                val_acc_history.append(epoch_acc)\n",
    "                valid_losses.append(epoch_loss)\n",
    "                scheduler.step()\n",
    "            if phase == 'train':\n",
    "                train_acc_history.append(epoch_acc)\n",
    "                train_losses.append(epoch_loss)\n",
    "        print('Optimizer learning rate : {:.7f}'.format(optimizer.param_groups[0]['lr']))\n",
    "        LRs.append(optimizer.param_groups[0]['lr'])\n",
    "\n",
    "    time_elapsed = time.time() - start_time\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "    # 训练完后用最好的一次当做模型最终的结果\n",
    "    model.load_state_dict(best_model_weights)\n",
    "    return model, val_acc_history, train_acc_history, valid_losses, train_losses, LRs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "873112ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b28e84f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA is available!  Training on GPU ...\n",
      "Params need to learn:\n",
      "fc.0.weight\n",
      "fc.0.bias\n",
      "['000none', '001wanpan', '002qiangzhuangnie', '003qianbijing', '004pingnie', '005zhiliaowan', '006kuiyinqi', '007tanzhen', '008kuoyinbang', '009huiyinxiaoduqian', '010shuchiqian', '011guashi', '012huojianqian', '013guchuanzhen', '014dabujinqian', '015xiaobujinqian', '016hezi', '017luanyuanqian', '018xiaoyaobei', '019wanqian', '020yahenban', '021jiandao', '022chizhenqi', '023chuancizhen']\n",
      "dataset_sizes below:\n",
      "{'train': 7056, 'valid': 4176}\n",
      "['000none', '001wanpan', '002qiangzhuangnie', '003qianbijing', '004pingnie', '005zhiliaowan', '006kuiyinqi', '007tanzhen', '008kuoyinbang', '009huiyinxiaoduqian', '010shuchiqian', '011guashi', '012huojianqian', '013guchuanzhen', '014dabujinqian', '015xiaobujinqian', '016hezi', '017luanyuanqian', '018xiaoyaobei', '019wanqian', '020yahenban', '021jiandao', '022chizhenqi', '023chuancizhen']\n",
      "Epoch 0/120\n",
      "----------\n",
      "Time elapsed 0m 30s\n",
      "train Loss: 3.8459 Acc: 0.6302\n",
      "Time elapsed 0m 39s\n",
      "valid Loss: 1.0172 Acc: 0.8408\n",
      "Optimizer learning rate : 0.0200000\n",
      "Epoch 1/120\n",
      "----------\n",
      "Time elapsed 1m 8s\n",
      "train Loss: 2.2401 Acc: 0.7690\n",
      "Time elapsed 1m 17s\n",
      "valid Loss: 4.3819 Acc: 0.7423\n",
      "Optimizer learning rate : 0.0200000\n",
      "Epoch 2/120\n",
      "----------\n",
      "Time elapsed 1m 46s\n",
      "train Loss: 2.6091 Acc: 0.7849\n",
      "Time elapsed 1m 55s\n",
      "valid Loss: 1.0286 Acc: 0.8884\n",
      "Optimizer learning rate : 0.0200000\n",
      "Epoch 3/120\n",
      "----------\n",
      "Time elapsed 2m 29s\n",
      "train Loss: 2.0940 Acc: 0.8253\n",
      "Time elapsed 2m 38s\n",
      "valid Loss: 1.6087 Acc: 0.8810\n",
      "Optimizer learning rate : 0.0160000\n",
      "Epoch 4/120\n",
      "----------\n",
      "Time elapsed 3m 12s\n",
      "train Loss: 1.9935 Acc: 0.8442\n",
      "Time elapsed 3m 21s\n",
      "valid Loss: 1.0209 Acc: 0.8970\n",
      "Optimizer learning rate : 0.0160000\n",
      "Epoch 5/120\n",
      "----------\n",
      "Time elapsed 3m 55s\n",
      "train Loss: 1.8713 Acc: 0.8539\n",
      "Time elapsed 4m 4s\n",
      "valid Loss: 3.5832 Acc: 0.8654\n",
      "Optimizer learning rate : 0.0160000\n",
      "Epoch 6/120\n",
      "----------\n",
      "Time elapsed 4m 38s\n",
      "train Loss: 1.5685 Acc: 0.8713\n",
      "Time elapsed 4m 47s\n",
      "valid Loss: 1.1316 Acc: 0.9028\n",
      "Optimizer learning rate : 0.0160000\n",
      "Epoch 7/120\n",
      "----------\n",
      "Time elapsed 5m 24s\n",
      "train Loss: 1.8318 Acc: 0.8621\n",
      "Time elapsed 5m 34s\n",
      "valid Loss: 3.9530 Acc: 0.8410\n",
      "Optimizer learning rate : 0.0128000\n",
      "Epoch 8/120\n",
      "----------\n",
      "Time elapsed 6m 10s\n",
      "train Loss: 1.4330 Acc: 0.8814\n",
      "Time elapsed 6m 20s\n",
      "valid Loss: 1.0781 Acc: 0.9083\n",
      "Optimizer learning rate : 0.0128000\n",
      "Epoch 9/120\n",
      "----------\n",
      "Time elapsed 6m 56s\n",
      "train Loss: 1.5132 Acc: 0.8849\n",
      "Time elapsed 7m 6s\n",
      "valid Loss: 0.9829 Acc: 0.9148\n",
      "Optimizer learning rate : 0.0128000\n",
      "Epoch 10/120\n",
      "----------\n",
      "Time elapsed 7m 43s\n",
      "train Loss: 1.4526 Acc: 0.8845\n",
      "Time elapsed 7m 53s\n",
      "valid Loss: 2.0561 Acc: 0.8934\n",
      "Optimizer learning rate : 0.0128000\n",
      "Epoch 11/120\n",
      "----------\n",
      "Time elapsed 8m 29s\n",
      "train Loss: 1.4828 Acc: 0.8895\n",
      "Time elapsed 8m 39s\n",
      "valid Loss: 0.5627 Acc: 0.9397\n",
      "Optimizer learning rate : 0.0102400\n",
      "Epoch 12/120\n",
      "----------\n",
      "Time elapsed 9m 16s\n",
      "train Loss: 1.2074 Acc: 0.8961\n",
      "Time elapsed 9m 26s\n",
      "valid Loss: 0.7594 Acc: 0.9258\n",
      "Optimizer learning rate : 0.0102400\n",
      "Epoch 13/120\n",
      "----------\n",
      "Time elapsed 10m 2s\n",
      "train Loss: 1.1637 Acc: 0.9031\n",
      "Time elapsed 10m 12s\n",
      "valid Loss: 0.5182 Acc: 0.9473\n",
      "Optimizer learning rate : 0.0102400\n",
      "Epoch 14/120\n",
      "----------\n",
      "Time elapsed 10m 48s\n",
      "train Loss: 1.0530 Acc: 0.9137\n",
      "Time elapsed 10m 58s\n",
      "valid Loss: 0.5656 Acc: 0.9468\n",
      "Optimizer learning rate : 0.0102400\n",
      "Epoch 15/120\n",
      "----------\n",
      "Time elapsed 11m 35s\n",
      "train Loss: 1.1229 Acc: 0.9073\n",
      "Time elapsed 11m 45s\n",
      "valid Loss: 0.6772 Acc: 0.9349\n",
      "Optimizer learning rate : 0.0081920\n",
      "Epoch 16/120\n",
      "----------\n",
      "Time elapsed 12m 21s\n",
      "train Loss: 0.8404 Acc: 0.9242\n",
      "Time elapsed 12m 31s\n",
      "valid Loss: 0.6358 Acc: 0.9365\n",
      "Optimizer learning rate : 0.0081920\n",
      "Epoch 17/120\n",
      "----------\n",
      "Time elapsed 13m 7s\n",
      "train Loss: 0.8783 Acc: 0.9211\n",
      "Time elapsed 13m 17s\n",
      "valid Loss: 1.4577 Acc: 0.9265\n",
      "Optimizer learning rate : 0.0081920\n",
      "Epoch 18/120\n",
      "----------\n",
      "Time elapsed 13m 53s\n",
      "train Loss: 0.8695 Acc: 0.9229\n",
      "Time elapsed 14m 3s\n",
      "valid Loss: 3.6971 Acc: 0.8982\n",
      "Optimizer learning rate : 0.0081920\n",
      "Epoch 19/120\n",
      "----------\n",
      "Time elapsed 14m 40s\n",
      "train Loss: 0.9180 Acc: 0.9158\n",
      "Time elapsed 14m 49s\n",
      "valid Loss: 0.3344 Acc: 0.9583\n",
      "Optimizer learning rate : 0.0065536\n",
      "Epoch 20/120\n",
      "----------\n",
      "Time elapsed 15m 23s\n",
      "train Loss: 0.7143 Acc: 0.9297\n",
      "Time elapsed 15m 32s\n",
      "valid Loss: 2.3106 Acc: 0.9255\n",
      "Optimizer learning rate : 0.0065536\n",
      "Epoch 21/120\n",
      "----------\n",
      "Time elapsed 16m 6s\n",
      "train Loss: 0.7906 Acc: 0.9236\n",
      "Time elapsed 16m 15s\n",
      "valid Loss: 2.3881 Acc: 0.9009\n",
      "Optimizer learning rate : 0.0065536\n",
      "Epoch 22/120\n",
      "----------\n",
      "Time elapsed 16m 48s\n",
      "train Loss: 0.7683 Acc: 0.9229\n",
      "Time elapsed 16m 58s\n",
      "valid Loss: 0.8966 Acc: 0.9471\n",
      "Optimizer learning rate : 0.0065536\n",
      "Epoch 23/120\n",
      "----------\n",
      "Time elapsed 17m 31s\n",
      "train Loss: 0.7978 Acc: 0.9269\n",
      "Time elapsed 17m 40s\n",
      "valid Loss: 0.2686 Acc: 0.9636\n",
      "Optimizer learning rate : 0.0052429\n",
      "Epoch 24/120\n",
      "----------\n",
      "Time elapsed 18m 14s\n",
      "train Loss: 0.6058 Acc: 0.9410\n",
      "Time elapsed 18m 24s\n",
      "valid Loss: 0.5105 Acc: 0.9447\n",
      "Optimizer learning rate : 0.0052429\n",
      "Epoch 25/120\n",
      "----------\n",
      "Time elapsed 18m 57s\n",
      "train Loss: 0.5890 Acc: 0.9366\n",
      "Time elapsed 19m 7s\n",
      "valid Loss: 0.9750 Acc: 0.9464\n",
      "Optimizer learning rate : 0.0052429\n",
      "Epoch 26/120\n",
      "----------\n",
      "Time elapsed 19m 40s\n",
      "train Loss: 0.6207 Acc: 0.9347\n",
      "Time elapsed 19m 50s\n",
      "valid Loss: 0.3601 Acc: 0.9574\n",
      "Optimizer learning rate : 0.0052429\n",
      "Epoch 27/120\n",
      "----------\n",
      "Time elapsed 20m 23s\n",
      "train Loss: 0.5784 Acc: 0.9365\n",
      "Time elapsed 20m 33s\n",
      "valid Loss: 0.8336 Acc: 0.9303\n",
      "Optimizer learning rate : 0.0041943\n",
      "Epoch 28/120\n",
      "----------\n",
      "Time elapsed 21m 6s\n",
      "train Loss: 0.5692 Acc: 0.9406\n",
      "Time elapsed 21m 16s\n",
      "valid Loss: 0.3178 Acc: 0.9583\n",
      "Optimizer learning rate : 0.0041943\n",
      "Epoch 29/120\n",
      "----------\n",
      "Time elapsed 21m 49s\n",
      "train Loss: 0.5990 Acc: 0.9364\n",
      "Time elapsed 21m 59s\n",
      "valid Loss: 0.3594 Acc: 0.9576\n",
      "Optimizer learning rate : 0.0041943\n",
      "Epoch 30/120\n",
      "----------\n",
      "Time elapsed 22m 32s\n",
      "train Loss: 0.4688 Acc: 0.9452\n",
      "Time elapsed 22m 42s\n",
      "valid Loss: 0.3153 Acc: 0.9643\n",
      "Optimizer learning rate : 0.0041943\n",
      "Epoch 31/120\n",
      "----------\n",
      "Time elapsed 23m 15s\n",
      "train Loss: 0.5734 Acc: 0.9366\n",
      "Time elapsed 23m 25s\n",
      "valid Loss: 0.4062 Acc: 0.9605\n",
      "Optimizer learning rate : 0.0033554\n",
      "Epoch 32/120\n",
      "----------\n",
      "Time elapsed 23m 58s\n",
      "train Loss: 0.4629 Acc: 0.9405\n",
      "Time elapsed 24m 8s\n",
      "valid Loss: 0.3256 Acc: 0.9670\n",
      "Optimizer learning rate : 0.0033554\n",
      "Epoch 33/120\n",
      "----------\n",
      "Time elapsed 24m 44s\n",
      "train Loss: 0.4863 Acc: 0.9453\n",
      "Time elapsed 24m 54s\n",
      "valid Loss: 0.3179 Acc: 0.9595\n",
      "Optimizer learning rate : 0.0033554\n",
      "Epoch 34/120\n",
      "----------\n",
      "Time elapsed 25m 27s\n",
      "train Loss: 0.4550 Acc: 0.9473\n",
      "Time elapsed 25m 36s\n",
      "valid Loss: 1.8482 Acc: 0.9423\n",
      "Optimizer learning rate : 0.0033554\n",
      "Epoch 35/120\n",
      "----------\n",
      "Time elapsed 26m 5s\n",
      "train Loss: 0.4655 Acc: 0.9422\n",
      "Time elapsed 26m 14s\n",
      "valid Loss: 0.7473 Acc: 0.9495\n",
      "Optimizer learning rate : 0.0026844\n",
      "Epoch 36/120\n",
      "----------\n",
      "Time elapsed 26m 43s\n",
      "train Loss: 0.4340 Acc: 0.9486\n",
      "Time elapsed 26m 52s\n",
      "valid Loss: 0.2271 Acc: 0.9691\n",
      "Optimizer learning rate : 0.0026844\n",
      "Epoch 37/120\n",
      "----------\n",
      "Time elapsed 27m 26s\n",
      "train Loss: 0.4114 Acc: 0.9487\n",
      "Time elapsed 27m 35s\n",
      "valid Loss: 2.5660 Acc: 0.9313\n",
      "Optimizer learning rate : 0.0026844\n",
      "Epoch 38/120\n",
      "----------\n",
      "Time elapsed 28m 9s\n",
      "train Loss: 0.3777 Acc: 0.9529\n",
      "Time elapsed 28m 18s\n",
      "valid Loss: 0.2549 Acc: 0.9665\n",
      "Optimizer learning rate : 0.0026844\n",
      "Epoch 39/120\n",
      "----------\n",
      "Time elapsed 28m 51s\n",
      "train Loss: 0.4499 Acc: 0.9459\n",
      "Time elapsed 29m 1s\n",
      "valid Loss: 0.2235 Acc: 0.9741\n",
      "Optimizer learning rate : 0.0021475\n",
      "Epoch 40/120\n",
      "----------\n",
      "Time elapsed 29m 30s\n",
      "train Loss: 0.3710 Acc: 0.9537\n",
      "Time elapsed 29m 39s\n",
      "valid Loss: 0.1621 Acc: 0.9739\n",
      "Optimizer learning rate : 0.0021475\n",
      "Epoch 41/120\n",
      "----------\n",
      "Time elapsed 30m 8s\n",
      "train Loss: 0.3699 Acc: 0.9538\n",
      "Time elapsed 30m 18s\n",
      "valid Loss: 3.2228 Acc: 0.9385\n",
      "Optimizer learning rate : 0.0021475\n",
      "Epoch 42/120\n",
      "----------\n",
      "Time elapsed 30m 50s\n",
      "train Loss: 0.3309 Acc: 0.9569\n",
      "Time elapsed 30m 59s\n",
      "valid Loss: 0.4595 Acc: 0.9636\n",
      "Optimizer learning rate : 0.0021475\n",
      "Epoch 43/120\n",
      "----------\n",
      "Time elapsed 31m 33s\n",
      "train Loss: 0.3694 Acc: 0.9542\n",
      "Time elapsed 31m 42s\n",
      "valid Loss: 0.1923 Acc: 0.9679\n",
      "Optimizer learning rate : 0.0017180\n",
      "Epoch 44/120\n",
      "----------\n",
      "Time elapsed 32m 16s\n",
      "train Loss: 0.2998 Acc: 0.9599\n",
      "Time elapsed 32m 25s\n",
      "valid Loss: 1.2652 Acc: 0.9502\n",
      "Optimizer learning rate : 0.0017180\n",
      "Epoch 45/120\n",
      "----------\n",
      "Time elapsed 32m 59s\n",
      "train Loss: 0.3010 Acc: 0.9593\n",
      "Time elapsed 33m 8s\n",
      "valid Loss: 7.4811 Acc: 0.9315\n",
      "Optimizer learning rate : 0.0017180\n",
      "Epoch 46/120\n",
      "----------\n",
      "Time elapsed 33m 42s\n",
      "train Loss: 0.3029 Acc: 0.9588\n",
      "Time elapsed 33m 51s\n",
      "valid Loss: 0.1953 Acc: 0.9756\n",
      "Optimizer learning rate : 0.0017180\n",
      "Epoch 47/120\n",
      "----------\n",
      "Time elapsed 34m 28s\n",
      "train Loss: 0.2884 Acc: 0.9626\n",
      "Time elapsed 34m 38s\n",
      "valid Loss: 0.6765 Acc: 0.9619\n",
      "Optimizer learning rate : 0.0013744\n",
      "Epoch 48/120\n",
      "----------\n",
      "Time elapsed 35m 15s\n",
      "train Loss: 0.2997 Acc: 0.9573\n",
      "Time elapsed 35m 25s\n",
      "valid Loss: 0.2109 Acc: 0.9684\n",
      "Optimizer learning rate : 0.0013744\n",
      "Epoch 49/120\n",
      "----------\n",
      "Time elapsed 36m 1s\n",
      "train Loss: 0.2593 Acc: 0.9578\n",
      "Time elapsed 36m 11s\n",
      "valid Loss: 0.1672 Acc: 0.9789\n",
      "Optimizer learning rate : 0.0013744\n",
      "Epoch 50/120\n",
      "----------\n",
      "Time elapsed 36m 48s\n",
      "train Loss: 0.2751 Acc: 0.9615\n",
      "Time elapsed 36m 58s\n",
      "valid Loss: 0.1748 Acc: 0.9725\n",
      "Optimizer learning rate : 0.0013744\n",
      "Epoch 51/120\n",
      "----------\n",
      "Time elapsed 37m 34s\n",
      "train Loss: 0.2681 Acc: 0.9590\n",
      "Time elapsed 37m 44s\n",
      "valid Loss: 0.1646 Acc: 0.9746\n",
      "Optimizer learning rate : 0.0010995\n",
      "Epoch 52/120\n",
      "----------\n",
      "Time elapsed 38m 21s\n",
      "train Loss: 0.2735 Acc: 0.9640\n",
      "Time elapsed 38m 31s\n",
      "valid Loss: 0.1541 Acc: 0.9768\n",
      "Optimizer learning rate : 0.0010995\n",
      "Epoch 53/120\n",
      "----------\n",
      "Time elapsed 39m 7s\n",
      "train Loss: 0.3004 Acc: 0.9589\n",
      "Time elapsed 39m 17s\n",
      "valid Loss: 0.1098 Acc: 0.9808\n",
      "Optimizer learning rate : 0.0010995\n",
      "Epoch 54/120\n",
      "----------\n",
      "Time elapsed 39m 54s\n",
      "train Loss: 0.2581 Acc: 0.9615\n",
      "Time elapsed 40m 4s\n",
      "valid Loss: 0.1568 Acc: 0.9768\n",
      "Optimizer learning rate : 0.0010995\n",
      "Epoch 55/120\n",
      "----------\n",
      "Time elapsed 40m 41s\n",
      "train Loss: 0.2304 Acc: 0.9656\n",
      "Time elapsed 40m 50s\n",
      "valid Loss: 0.1654 Acc: 0.9739\n",
      "Optimizer learning rate : 0.0008796\n",
      "Epoch 56/120\n",
      "----------\n",
      "Time elapsed 41m 27s\n",
      "train Loss: 0.2358 Acc: 0.9650\n",
      "Time elapsed 41m 37s\n",
      "valid Loss: 0.1315 Acc: 0.9813\n",
      "Optimizer learning rate : 0.0008796\n",
      "Epoch 57/120\n",
      "----------\n",
      "Time elapsed 42m 14s\n",
      "train Loss: 0.2385 Acc: 0.9632\n",
      "Time elapsed 42m 24s\n",
      "valid Loss: 0.1385 Acc: 0.9744\n",
      "Optimizer learning rate : 0.0008796\n",
      "Epoch 58/120\n",
      "----------\n",
      "Time elapsed 43m 0s\n",
      "train Loss: 0.2300 Acc: 0.9656\n",
      "Time elapsed 43m 10s\n",
      "valid Loss: 0.1249 Acc: 0.9763\n",
      "Optimizer learning rate : 0.0008796\n",
      "Epoch 59/120\n",
      "----------\n",
      "Time elapsed 43m 47s\n",
      "train Loss: 0.2140 Acc: 0.9677\n",
      "Time elapsed 43m 57s\n",
      "valid Loss: 0.6114 Acc: 0.9638\n",
      "Optimizer learning rate : 0.0007037\n",
      "Epoch 60/120\n",
      "----------\n",
      "Time elapsed 44m 33s\n",
      "train Loss: 0.2593 Acc: 0.9606\n",
      "Time elapsed 44m 43s\n",
      "valid Loss: 0.1604 Acc: 0.9792\n",
      "Optimizer learning rate : 0.0007037\n",
      "Epoch 61/120\n",
      "----------\n",
      "Time elapsed 45m 20s\n",
      "train Loss: 0.2423 Acc: 0.9624\n",
      "Time elapsed 45m 29s\n",
      "valid Loss: 0.0959 Acc: 0.9818\n",
      "Optimizer learning rate : 0.0007037\n",
      "Epoch 62/120\n",
      "----------\n",
      "Time elapsed 45m 59s\n",
      "train Loss: 0.2084 Acc: 0.9657\n",
      "Time elapsed 46m 8s\n",
      "valid Loss: 1.6269 Acc: 0.9480\n",
      "Optimizer learning rate : 0.0007037\n",
      "Epoch 63/120\n",
      "----------\n",
      "Time elapsed 46m 37s\n",
      "train Loss: 0.2449 Acc: 0.9616\n",
      "Time elapsed 46m 47s\n",
      "valid Loss: 1.0946 Acc: 0.9595\n",
      "Optimizer learning rate : 0.0005629\n",
      "Epoch 64/120\n",
      "----------\n",
      "Time elapsed 47m 15s\n",
      "train Loss: 0.2259 Acc: 0.9641\n",
      "Time elapsed 47m 25s\n",
      "valid Loss: 4.1995 Acc: 0.9365\n",
      "Optimizer learning rate : 0.0005629\n",
      "Epoch 65/120\n",
      "----------\n",
      "Time elapsed 47m 54s\n",
      "train Loss: 0.2425 Acc: 0.9636\n",
      "Time elapsed 48m 3s\n",
      "valid Loss: 0.1293 Acc: 0.9787\n",
      "Optimizer learning rate : 0.0005629\n",
      "Epoch 66/120\n",
      "----------\n",
      "Time elapsed 48m 32s\n",
      "train Loss: 0.2141 Acc: 0.9697\n",
      "Time elapsed 48m 41s\n",
      "valid Loss: 0.1344 Acc: 0.9775\n",
      "Optimizer learning rate : 0.0005629\n",
      "Epoch 67/120\n",
      "----------\n",
      "Time elapsed 49m 10s\n",
      "train Loss: 0.2465 Acc: 0.9623\n",
      "Time elapsed 49m 19s\n",
      "valid Loss: 2.7788 Acc: 0.9418\n",
      "Optimizer learning rate : 0.0004504\n",
      "Epoch 68/120\n",
      "----------\n",
      "Time elapsed 49m 48s\n",
      "train Loss: 0.2300 Acc: 0.9654\n",
      "Time elapsed 49m 58s\n",
      "valid Loss: 0.1238 Acc: 0.9761\n",
      "Optimizer learning rate : 0.0004504\n",
      "Epoch 69/120\n",
      "----------\n",
      "Time elapsed 50m 27s\n",
      "train Loss: 0.1906 Acc: 0.9678\n",
      "Time elapsed 50m 36s\n",
      "valid Loss: 0.1724 Acc: 0.9799\n",
      "Optimizer learning rate : 0.0004504\n",
      "Epoch 70/120\n",
      "----------\n",
      "Time elapsed 51m 5s\n",
      "train Loss: 0.2167 Acc: 0.9667\n",
      "Time elapsed 51m 14s\n",
      "valid Loss: 1.1035 Acc: 0.9586\n",
      "Optimizer learning rate : 0.0004504\n",
      "Epoch 71/120\n",
      "----------\n",
      "Time elapsed 51m 43s\n",
      "train Loss: 0.1713 Acc: 0.9719\n",
      "Time elapsed 51m 52s\n",
      "valid Loss: 0.1094 Acc: 0.9784\n",
      "Optimizer learning rate : 0.0003603\n",
      "Epoch 72/120\n",
      "----------\n",
      "Time elapsed 52m 21s\n",
      "train Loss: 0.2040 Acc: 0.9658\n",
      "Time elapsed 52m 31s\n",
      "valid Loss: 0.1004 Acc: 0.9804\n",
      "Optimizer learning rate : 0.0003603\n",
      "Epoch 73/120\n",
      "----------\n",
      "Time elapsed 52m 59s\n",
      "train Loss: 0.1803 Acc: 0.9678\n",
      "Time elapsed 53m 9s\n",
      "valid Loss: 3.6891 Acc: 0.9397\n",
      "Optimizer learning rate : 0.0003603\n",
      "Epoch 74/120\n",
      "----------\n",
      "Time elapsed 53m 38s\n",
      "train Loss: 0.1900 Acc: 0.9707\n",
      "Time elapsed 53m 47s\n",
      "valid Loss: 4.0048 Acc: 0.9404\n",
      "Optimizer learning rate : 0.0003603\n",
      "Epoch 75/120\n",
      "----------\n",
      "Time elapsed 54m 16s\n",
      "train Loss: 0.1937 Acc: 0.9675\n",
      "Time elapsed 54m 25s\n",
      "valid Loss: 2.2934 Acc: 0.9485\n",
      "Optimizer learning rate : 0.0002882\n",
      "Epoch 76/120\n",
      "----------\n",
      "Time elapsed 54m 54s\n",
      "train Loss: 0.1789 Acc: 0.9688\n",
      "Time elapsed 55m 3s\n",
      "valid Loss: 0.0983 Acc: 0.9801\n",
      "Optimizer learning rate : 0.0002882\n",
      "Epoch 77/120\n",
      "----------\n",
      "Time elapsed 55m 32s\n",
      "train Loss: 0.2191 Acc: 0.9664\n",
      "Time elapsed 55m 41s\n",
      "valid Loss: 4.2382 Acc: 0.9373\n",
      "Optimizer learning rate : 0.0002882\n",
      "Epoch 78/120\n",
      "----------\n",
      "Time elapsed 56m 10s\n",
      "train Loss: 0.1961 Acc: 0.9658\n",
      "Time elapsed 56m 19s\n",
      "valid Loss: 0.1002 Acc: 0.9782\n",
      "Optimizer learning rate : 0.0002882\n",
      "Epoch 79/120\n",
      "----------\n",
      "Time elapsed 56m 48s\n",
      "train Loss: 0.1689 Acc: 0.9704\n",
      "Time elapsed 56m 57s\n",
      "valid Loss: 1.4221 Acc: 0.9564\n",
      "Optimizer learning rate : 0.0002306\n",
      "Epoch 80/120\n",
      "----------\n",
      "Time elapsed 57m 26s\n",
      "train Loss: 0.1943 Acc: 0.9658\n",
      "Time elapsed 57m 35s\n",
      "valid Loss: 0.1298 Acc: 0.9775\n",
      "Optimizer learning rate : 0.0002306\n",
      "Epoch 81/120\n",
      "----------\n",
      "Time elapsed 58m 4s\n",
      "train Loss: 0.2051 Acc: 0.9694\n",
      "Time elapsed 58m 13s\n",
      "valid Loss: 0.2652 Acc: 0.9763\n",
      "Optimizer learning rate : 0.0002306\n",
      "Epoch 82/120\n",
      "----------\n",
      "Time elapsed 58m 42s\n",
      "train Loss: 0.1952 Acc: 0.9688\n",
      "Time elapsed 58m 52s\n",
      "valid Loss: 2.9865 Acc: 0.9394\n",
      "Optimizer learning rate : 0.0002306\n",
      "Epoch 83/120\n",
      "----------\n",
      "Time elapsed 59m 20s\n",
      "train Loss: 0.1638 Acc: 0.9687\n",
      "Time elapsed 59m 30s\n",
      "valid Loss: 3.1264 Acc: 0.9389\n",
      "Optimizer learning rate : 0.0001845\n",
      "Epoch 84/120\n",
      "----------\n",
      "Time elapsed 59m 58s\n",
      "train Loss: 0.1815 Acc: 0.9692\n",
      "Time elapsed 60m 8s\n",
      "valid Loss: 0.9043 Acc: 0.9631\n",
      "Optimizer learning rate : 0.0001845\n",
      "Epoch 85/120\n",
      "----------\n",
      "Time elapsed 60m 36s\n",
      "train Loss: 0.1770 Acc: 0.9695\n",
      "Time elapsed 60m 46s\n",
      "valid Loss: 0.3448 Acc: 0.9725\n",
      "Optimizer learning rate : 0.0001845\n",
      "Epoch 86/120\n",
      "----------\n",
      "Time elapsed 61m 14s\n",
      "train Loss: 0.1842 Acc: 0.9692\n",
      "Time elapsed 61m 24s\n",
      "valid Loss: 0.1319 Acc: 0.9801\n",
      "Optimizer learning rate : 0.0001845\n",
      "Epoch 87/120\n",
      "----------\n",
      "Time elapsed 61m 52s\n",
      "train Loss: 0.1646 Acc: 0.9707\n",
      "Time elapsed 62m 2s\n",
      "valid Loss: 0.2774 Acc: 0.9765\n",
      "Optimizer learning rate : 0.0001476\n",
      "Epoch 88/120\n",
      "----------\n",
      "Time elapsed 62m 31s\n",
      "train Loss: 0.1680 Acc: 0.9698\n",
      "Time elapsed 62m 40s\n",
      "valid Loss: 0.1838 Acc: 0.9799\n",
      "Optimizer learning rate : 0.0001476\n",
      "Epoch 89/120\n",
      "----------\n",
      "Time elapsed 63m 9s\n",
      "train Loss: 0.1877 Acc: 0.9700\n",
      "Time elapsed 63m 18s\n",
      "valid Loss: 0.8602 Acc: 0.9607\n",
      "Optimizer learning rate : 0.0001476\n",
      "Epoch 90/120\n",
      "----------\n",
      "Time elapsed 63m 47s\n",
      "train Loss: 0.1846 Acc: 0.9678\n",
      "Time elapsed 63m 56s\n",
      "valid Loss: 0.0937 Acc: 0.9818\n",
      "Optimizer learning rate : 0.0001476\n",
      "Epoch 91/120\n",
      "----------\n",
      "Time elapsed 64m 25s\n",
      "train Loss: 0.1710 Acc: 0.9718\n",
      "Time elapsed 64m 34s\n",
      "valid Loss: 1.5111 Acc: 0.9557\n",
      "Optimizer learning rate : 0.0001181\n",
      "Epoch 92/120\n",
      "----------\n",
      "Time elapsed 65m 3s\n",
      "train Loss: 0.1829 Acc: 0.9688\n",
      "Time elapsed 65m 12s\n",
      "valid Loss: 0.1090 Acc: 0.9813\n",
      "Optimizer learning rate : 0.0001181\n",
      "Epoch 93/120\n",
      "----------\n",
      "Time elapsed 65m 41s\n",
      "train Loss: 0.1921 Acc: 0.9715\n",
      "Time elapsed 65m 50s\n",
      "valid Loss: 1.6559 Acc: 0.9557\n",
      "Optimizer learning rate : 0.0001181\n",
      "Epoch 94/120\n",
      "----------\n",
      "Time elapsed 66m 21s\n",
      "train Loss: 0.1480 Acc: 0.9722\n",
      "Time elapsed 66m 31s\n",
      "valid Loss: 0.6709 Acc: 0.9662\n",
      "Optimizer learning rate : 0.0001181\n",
      "Epoch 95/120\n",
      "----------\n",
      "Time elapsed 67m 4s\n",
      "train Loss: 0.2172 Acc: 0.9644\n",
      "Time elapsed 67m 14s\n",
      "valid Loss: 0.1163 Acc: 0.9801\n",
      "Optimizer learning rate : 0.0000944\n",
      "Epoch 96/120\n",
      "----------\n",
      "Time elapsed 67m 47s\n",
      "train Loss: 0.1649 Acc: 0.9698\n",
      "Time elapsed 67m 56s\n",
      "valid Loss: 0.1453 Acc: 0.9780\n",
      "Optimizer learning rate : 0.0000944\n",
      "Epoch 97/120\n",
      "----------\n",
      "Time elapsed 68m 30s\n",
      "train Loss: 0.2077 Acc: 0.9671\n",
      "Time elapsed 68m 39s\n",
      "valid Loss: 0.2357 Acc: 0.9787\n",
      "Optimizer learning rate : 0.0000944\n",
      "Epoch 98/120\n",
      "----------\n",
      "Time elapsed 69m 12s\n",
      "train Loss: 0.1847 Acc: 0.9707\n",
      "Time elapsed 69m 22s\n",
      "valid Loss: 0.1027 Acc: 0.9794\n",
      "Optimizer learning rate : 0.0000944\n",
      "Epoch 99/120\n",
      "----------\n",
      "Time elapsed 69m 55s\n",
      "train Loss: 0.2018 Acc: 0.9668\n",
      "Time elapsed 70m 5s\n",
      "valid Loss: 0.0887 Acc: 0.9813\n",
      "Optimizer learning rate : 0.0000756\n",
      "Epoch 100/120\n",
      "----------\n",
      "Time elapsed 70m 38s\n",
      "train Loss: 0.1672 Acc: 0.9702\n",
      "Time elapsed 70m 48s\n",
      "valid Loss: 0.1110 Acc: 0.9811\n",
      "Optimizer learning rate : 0.0000756\n",
      "Epoch 101/120\n",
      "----------\n",
      "Time elapsed 71m 21s\n",
      "train Loss: 0.1755 Acc: 0.9722\n",
      "Time elapsed 71m 30s\n",
      "valid Loss: 0.1199 Acc: 0.9801\n",
      "Optimizer learning rate : 0.0000756\n",
      "Epoch 102/120\n",
      "----------\n",
      "Time elapsed 72m 4s\n",
      "train Loss: 0.1987 Acc: 0.9697\n",
      "Time elapsed 72m 13s\n",
      "valid Loss: 0.2308 Acc: 0.9753\n",
      "Optimizer learning rate : 0.0000756\n",
      "Epoch 103/120\n",
      "----------\n",
      "Time elapsed 72m 46s\n",
      "train Loss: 0.1805 Acc: 0.9711\n",
      "Time elapsed 72m 56s\n",
      "valid Loss: 0.1682 Acc: 0.9787\n",
      "Optimizer learning rate : 0.0000604\n",
      "Epoch 104/120\n",
      "----------\n",
      "Time elapsed 73m 29s\n",
      "train Loss: 0.1915 Acc: 0.9707\n",
      "Time elapsed 73m 39s\n",
      "valid Loss: 0.1183 Acc: 0.9799\n",
      "Optimizer learning rate : 0.0000604\n",
      "Epoch 105/120\n",
      "----------\n",
      "Time elapsed 74m 12s\n",
      "train Loss: 0.1894 Acc: 0.9702\n",
      "Time elapsed 74m 22s\n",
      "valid Loss: 0.3216 Acc: 0.9770\n",
      "Optimizer learning rate : 0.0000604\n",
      "Epoch 106/120\n",
      "----------\n",
      "Time elapsed 74m 55s\n",
      "train Loss: 0.1767 Acc: 0.9690\n",
      "Time elapsed 75m 5s\n",
      "valid Loss: 0.0881 Acc: 0.9811\n",
      "Optimizer learning rate : 0.0000604\n",
      "Epoch 107/120\n",
      "----------\n",
      "Time elapsed 75m 38s\n",
      "train Loss: 0.1661 Acc: 0.9701\n",
      "Time elapsed 75m 47s\n",
      "valid Loss: 2.9392 Acc: 0.9456\n",
      "Optimizer learning rate : 0.0000484\n",
      "Epoch 108/120\n",
      "----------\n",
      "Time elapsed 76m 21s\n",
      "train Loss: 0.1744 Acc: 0.9691\n",
      "Time elapsed 76m 30s\n",
      "valid Loss: 0.2923 Acc: 0.9780\n",
      "Optimizer learning rate : 0.0000484\n",
      "Epoch 109/120\n",
      "----------\n",
      "Time elapsed 77m 4s\n",
      "train Loss: 0.1796 Acc: 0.9695\n",
      "Time elapsed 77m 13s\n",
      "valid Loss: 0.1275 Acc: 0.9806\n",
      "Optimizer learning rate : 0.0000484\n",
      "Epoch 110/120\n",
      "----------\n",
      "Time elapsed 77m 46s\n",
      "train Loss: 0.1783 Acc: 0.9721\n",
      "Time elapsed 77m 56s\n",
      "valid Loss: 0.1542 Acc: 0.9794\n",
      "Optimizer learning rate : 0.0000484\n",
      "Epoch 111/120\n",
      "----------\n",
      "Time elapsed 78m 29s\n",
      "train Loss: 0.1811 Acc: 0.9688\n",
      "Time elapsed 78m 39s\n",
      "valid Loss: 1.7982 Acc: 0.9562\n",
      "Optimizer learning rate : 0.0000387\n",
      "Epoch 112/120\n",
      "----------\n",
      "Time elapsed 79m 13s\n",
      "train Loss: 0.1911 Acc: 0.9681\n",
      "Time elapsed 79m 22s\n",
      "valid Loss: 0.3041 Acc: 0.9768\n",
      "Optimizer learning rate : 0.0000387\n",
      "Epoch 113/120\n",
      "----------\n",
      "Time elapsed 79m 56s\n",
      "train Loss: 0.1922 Acc: 0.9675\n",
      "Time elapsed 80m 5s\n",
      "valid Loss: 1.9727 Acc: 0.9483\n",
      "Optimizer learning rate : 0.0000387\n",
      "Epoch 114/120\n",
      "----------\n",
      "Time elapsed 80m 39s\n",
      "train Loss: 0.1678 Acc: 0.9708\n",
      "Time elapsed 80m 49s\n",
      "valid Loss: 0.4247 Acc: 0.9756\n",
      "Optimizer learning rate : 0.0000387\n",
      "Epoch 115/120\n",
      "----------\n",
      "Time elapsed 81m 22s\n",
      "train Loss: 0.1603 Acc: 0.9734\n",
      "Time elapsed 81m 32s\n",
      "valid Loss: 1.2504 Acc: 0.9636\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 116/120\n",
      "----------\n",
      "Time elapsed 82m 5s\n",
      "train Loss: 0.1667 Acc: 0.9726\n",
      "Time elapsed 82m 15s\n",
      "valid Loss: 0.1050 Acc: 0.9825\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 117/120\n",
      "----------\n",
      "Time elapsed 82m 48s\n",
      "train Loss: 0.1515 Acc: 0.9726\n",
      "Time elapsed 82m 58s\n",
      "valid Loss: 0.1161 Acc: 0.9816\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 118/120\n",
      "----------\n",
      "Time elapsed 83m 31s\n",
      "train Loss: 0.1933 Acc: 0.9680\n",
      "Time elapsed 83m 41s\n",
      "valid Loss: 0.4359 Acc: 0.9720\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 119/120\n",
      "----------\n",
      "Time elapsed 84m 14s\n",
      "train Loss: 0.1780 Acc: 0.9721\n",
      "Time elapsed 84m 24s\n",
      "valid Loss: 0.2583 Acc: 0.9756\n",
      "Optimizer learning rate : 0.0000248\n",
      "Epoch 120/120\n",
      "----------\n",
      "Time elapsed 84m 57s\n",
      "train Loss: 0.1781 Acc: 0.9715\n",
      "Time elapsed 85m 7s\n",
      "valid Loss: 0.1700 Acc: 0.9787\n",
      "Optimizer learning rate : 0.0000248\n",
      "Training complete in 85m 7s\n",
      "Best val Acc: 0.982519\n",
      "Epoch 0/24\n",
      "----------\n",
      "Time elapsed 1m 8s\n",
      "train Loss: 0.1956 Acc: 0.9698\n",
      "Time elapsed 1m 18s\n",
      "valid Loss: 0.8937 Acc: 0.9648\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 1/24\n",
      "----------\n",
      "Time elapsed 2m 29s\n",
      "train Loss: 0.1902 Acc: 0.9680\n",
      "Time elapsed 2m 39s\n",
      "valid Loss: 0.1053 Acc: 0.9813\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 2/24\n",
      "----------\n",
      "Time elapsed 3m 45s\n",
      "train Loss: 0.1829 Acc: 0.9707\n",
      "Time elapsed 3m 55s\n",
      "valid Loss: 0.1031 Acc: 0.9789\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 3/24\n",
      "----------\n",
      "Time elapsed 5m 2s\n",
      "train Loss: 0.1698 Acc: 0.9715\n",
      "Time elapsed 5m 12s\n",
      "valid Loss: 1.9456 Acc: 0.9478\n",
      "Optimizer learning rate : 0.0000309\n",
      "Epoch 4/24\n",
      "----------\n",
      "Time elapsed 6m 19s\n",
      "train Loss: 0.1638 Acc: 0.9721\n",
      "Time elapsed 6m 29s\n",
      "valid Loss: 2.3843 Acc: 0.9468\n",
      "Optimizer learning rate : 0.0000155\n",
      "Epoch 5/24\n",
      "----------\n",
      "Time elapsed 7m 36s\n",
      "train Loss: 0.1662 Acc: 0.9697\n",
      "Time elapsed 7m 46s\n",
      "valid Loss: 0.1696 Acc: 0.9780\n",
      "Optimizer learning rate : 0.0000155\n",
      "Epoch 6/24\n",
      "----------\n",
      "Time elapsed 8m 53s\n",
      "train Loss: 0.1636 Acc: 0.9705\n",
      "Time elapsed 9m 2s\n",
      "valid Loss: 1.5067 Acc: 0.9535\n",
      "Optimizer learning rate : 0.0000155\n",
      "Epoch 7/24\n",
      "----------\n",
      "Time elapsed 10m 10s\n",
      "train Loss: 0.1836 Acc: 0.9694\n",
      "Time elapsed 10m 20s\n",
      "valid Loss: 0.1516 Acc: 0.9794\n",
      "Optimizer learning rate : 0.0000155\n",
      "Epoch 8/24\n",
      "----------\n",
      "Time elapsed 11m 28s\n",
      "train Loss: 0.1570 Acc: 0.9690\n",
      "Time elapsed 11m 38s\n",
      "valid Loss: 0.1295 Acc: 0.9794\n",
      "Optimizer learning rate : 0.0000155\n",
      "Epoch 9/24\n",
      "----------\n",
      "Time elapsed 12m 48s\n",
      "train Loss: 0.1688 Acc: 0.9709\n",
      "Time elapsed 12m 58s\n",
      "valid Loss: 0.1629 Acc: 0.9823\n",
      "Optimizer learning rate : 0.0000077\n",
      "Epoch 10/24\n",
      "----------\n",
      "Time elapsed 14m 6s\n",
      "train Loss: 0.2055 Acc: 0.9698\n",
      "Time elapsed 14m 15s\n",
      "valid Loss: 0.1711 Acc: 0.9794\n",
      "Optimizer learning rate : 0.0000077\n",
      "Epoch 11/24\n",
      "----------\n",
      "Time elapsed 15m 22s\n",
      "train Loss: 0.1733 Acc: 0.9717\n",
      "Time elapsed 15m 32s\n",
      "valid Loss: 3.1842 Acc: 0.9418\n",
      "Optimizer learning rate : 0.0000077\n",
      "Epoch 12/24\n",
      "----------\n",
      "Time elapsed 16m 39s\n",
      "train Loss: 0.1763 Acc: 0.9681\n",
      "Time elapsed 16m 49s\n",
      "valid Loss: 0.2995 Acc: 0.9746\n",
      "Optimizer learning rate : 0.0000077\n",
      "Epoch 13/24\n",
      "----------\n",
      "Time elapsed 17m 56s\n",
      "train Loss: 0.1709 Acc: 0.9700\n",
      "Time elapsed 18m 6s\n",
      "valid Loss: 0.1415 Acc: 0.9811\n",
      "Optimizer learning rate : 0.0000077\n",
      "Epoch 14/24\n",
      "----------\n",
      "Time elapsed 19m 13s\n",
      "train Loss: 0.1693 Acc: 0.9712\n",
      "Time elapsed 19m 23s\n",
      "valid Loss: 0.5729 Acc: 0.9713\n",
      "Optimizer learning rate : 0.0000039\n",
      "Epoch 15/24\n",
      "----------\n",
      "Time elapsed 20m 30s\n",
      "train Loss: 0.1941 Acc: 0.9715\n",
      "Time elapsed 20m 40s\n",
      "valid Loss: 0.5983 Acc: 0.9665\n",
      "Optimizer learning rate : 0.0000039\n",
      "Epoch 16/24\n",
      "----------\n",
      "Time elapsed 21m 47s\n",
      "train Loss: 0.2019 Acc: 0.9673\n",
      "Time elapsed 21m 57s\n",
      "valid Loss: 1.6472 Acc: 0.9559\n",
      "Optimizer learning rate : 0.0000039\n",
      "Epoch 17/24\n",
      "----------\n",
      "Time elapsed 23m 4s\n",
      "train Loss: 0.1751 Acc: 0.9695\n",
      "Time elapsed 23m 13s\n",
      "valid Loss: 0.1096 Acc: 0.9801\n",
      "Optimizer learning rate : 0.0000039\n",
      "Epoch 18/24\n",
      "----------\n",
      "Time elapsed 24m 21s\n",
      "train Loss: 0.1643 Acc: 0.9701\n",
      "Time elapsed 24m 30s\n",
      "valid Loss: 4.9583 Acc: 0.9385\n",
      "Optimizer learning rate : 0.0000039\n",
      "Epoch 19/24\n",
      "----------\n",
      "Time elapsed 25m 38s\n",
      "train Loss: 0.1776 Acc: 0.9673\n",
      "Time elapsed 25m 47s\n",
      "valid Loss: 0.1050 Acc: 0.9804\n",
      "Optimizer learning rate : 0.0000019\n",
      "Epoch 20/24\n",
      "----------\n",
      "Time elapsed 26m 54s\n",
      "train Loss: 0.1649 Acc: 0.9735\n",
      "Time elapsed 27m 4s\n",
      "valid Loss: 0.0952 Acc: 0.9820\n",
      "Optimizer learning rate : 0.0000019\n",
      "Epoch 21/24\n",
      "----------\n",
      "Time elapsed 28m 11s\n",
      "train Loss: 0.1726 Acc: 0.9692\n",
      "Time elapsed 28m 21s\n",
      "valid Loss: 1.2185 Acc: 0.9564\n",
      "Optimizer learning rate : 0.0000019\n",
      "Epoch 22/24\n",
      "----------\n",
      "Time elapsed 29m 28s\n",
      "train Loss: 0.1726 Acc: 0.9714\n",
      "Time elapsed 29m 38s\n",
      "valid Loss: 0.2181 Acc: 0.9777\n",
      "Optimizer learning rate : 0.0000019\n",
      "Epoch 23/24\n",
      "----------\n",
      "Time elapsed 30m 45s\n",
      "train Loss: 0.1554 Acc: 0.9707\n",
      "Time elapsed 30m 55s\n",
      "valid Loss: 0.1077 Acc: 0.9806\n",
      "Optimizer learning rate : 0.0000019\n",
      "Epoch 24/24\n",
      "----------\n",
      "Time elapsed 32m 2s\n",
      "train Loss: 0.1649 Acc: 0.9722\n",
      "Time elapsed 32m 12s\n",
      "valid Loss: 0.2087 Acc: 0.9782\n",
      "Optimizer learning rate : 0.0000010\n",
      "Training complete in 32m 12s\n",
      "Best val Acc: 0.982280\n"
     ]
    }
   ],
   "source": [
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "lab_start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6c90f9a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
